package com.jusyl.cristo.spider.smzdm;

import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.List;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

public class SmzdmProcessor implements PageProcessor{

	private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);
	private static List<String> urlList = new ArrayList<String>();
	
	public SmzdmProcessor(){
	}

	@Override
	public void process(Page page) {
		//获取页面信息以供分析参考
		System.out.println("开始进入Page");

		
		List<String> strs = page.getHtml().xpath("//div[@class='z-feed-img']/a").all();
		for(String s : strs) {
			//寻找匹配的关键词
			if(!s.contains("小米")  || urlList.size() > 10000)
				continue;
			
			int herfIndex = s.indexOf("href");
			int targetIndex = s.indexOf("target");
			//System.out.println(herfIndex);
			//System.out.println(targetIndex);
			
			String subString = s.substring(herfIndex+6, targetIndex-2);
			if(subString.contains("https")) {
				System.out.println(subString);
				urlList.add(subString);
			}
			
			
		}
		
		System.out.println("开始整下一页的内容");
		
		List<String> targetList = page.getHtml().xpath("//ul[@class='pagenation-list']/li/html()").all();
		
		System.out.println(targetList.size());
		//首页即为第一页，所以后面的工作从第二页开始
		for(int i = 1; i < 3; i++ ) {
			String targetStr = targetList.get(i);
			int startIndex = targetStr.indexOf("href") + 6;
			int endIndex = targetStr.indexOf("\">");
			
			System.out.println(startIndex);
			System.out.println(endIndex);
			//System.out.println("https:" + targetStr.substring(startIndex, endIndex));
			String herf = "https:" + targetStr.substring(startIndex, endIndex);
			try {
				System.out.println(URLDecoder.decode(herf, "UTF-8"));
				page.addTargetRequest(URLDecoder.decode(herf, "UTF-8").replaceAll("&amp;", "&"));
				
			} catch (UnsupportedEncodingException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
		

	}

	@Override
	public Site getSite() {
		return site;
	}
	
	public static void main(String arg[]) {
		Spider.create(new SmzdmProcessor())
			.addUrl("https://search.smzdm.com/?c=home&s=%E5%B0%8F%E7%B1%B3&order=time&v=b")
			.thread(1).run();
	}
}
